{
 "cells": [
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-16T05:55:39.721062Z",
     "start_time": "2024-11-16T05:55:37.777062Z"
    }
   },
   "cell_type": "code",
   "source": [
    "#  导入 requests 库\n",
    "import requests\n",
    "import os\n",
    "#  通过 requests 库get方法访问 http://www.olympedia.org/editions 网址，获取网页内容\n",
    "\n",
    "headers = {\n",
    "    'user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.131 Safari/537.36'\n",
    "}\n",
    "url = 'http://www.olympedia.org/editions'\n",
    "page_text = requests.get(url=url, headers=headers).text"
   ],
   "id": "809c61000f53e405",
   "outputs": [],
   "execution_count": 1
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-16T05:55:45.527963Z",
     "start_time": "2024-11-16T05:55:45.468392Z"
    }
   },
   "cell_type": "code",
   "source": [
    "from lxml import etree\n",
    "list1=[]\n",
    "#实例化etree对象\n",
    "tree = etree.HTML(page_text)\n",
    "#查找年份对应的标签\n",
    "#点开每一年对应的链接查看一下之后，会发现只是在官网链接后面添加了一个短链接，拼接在一起就是详细页面的完整链接了\n",
    "r1 = tree.xpath('/html/body/div[2]/table[1]//tr')\n",
    "for ii in r1:\n",
    "    r2 = ii.xpath('./td[1]/a/@href')#提取出详细页面的URL\n",
    "    # print(r2)\n",
    "    if r2:\n",
    "        for u in r2:\n",
    "            list1.append('http://www.olympedia.org' + u)#这里是拼接完整的链接，能直接进入到所要爬取的具体页面\n",
    "    #可以输出list1列表中的url查看获取的是否正确  \n",
    "print(list1)"
   ],
   "id": "dad1d3363b309454",
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['http://www.olympedia.org/editions/1', 'http://www.olympedia.org/editions/2', 'http://www.olympedia.org/editions/3', 'http://www.olympedia.org/editions/5', 'http://www.olympedia.org/editions/6', 'http://www.olympedia.org/editions/50', 'http://www.olympedia.org/editions/7', 'http://www.olympedia.org/editions/8', 'http://www.olympedia.org/editions/9', 'http://www.olympedia.org/editions/10', 'http://www.olympedia.org/editions/11', 'http://www.olympedia.org/editions/51', 'http://www.olympedia.org/editions/52', 'http://www.olympedia.org/editions/12', 'http://www.olympedia.org/editions/13', 'http://www.olympedia.org/editions/14', 'http://www.olympedia.org/editions/15', 'http://www.olympedia.org/editions/16', 'http://www.olympedia.org/editions/17', 'http://www.olympedia.org/editions/18', 'http://www.olympedia.org/editions/19', 'http://www.olympedia.org/editions/20', 'http://www.olympedia.org/editions/21', 'http://www.olympedia.org/editions/22', 'http://www.olympedia.org/editions/23', 'http://www.olympedia.org/editions/24', 'http://www.olympedia.org/editions/25', 'http://www.olympedia.org/editions/26', 'http://www.olympedia.org/editions/53', 'http://www.olympedia.org/editions/54', 'http://www.olympedia.org/editions/59', 'http://www.olympedia.org/editions/61', 'http://www.olympedia.org/editions/63', 'http://www.olympedia.org/editions/64', 'http://www.olympedia.org/editions/372']\n"
     ]
    }
   ],
   "execution_count": 2
  },
  {
   "metadata": {
    "ExecuteTime": {
     "end_time": "2024-11-09T07:50:42.749949Z",
     "start_time": "2024-11-09T07:49:58.952789Z"
    }
   },
   "cell_type": "code",
   "source": [
    "for i in range(len(list1)):\n",
    "    listurl=list1[i]\n",
    "    test1 = requests.get(url=listurl, headers=headers).text\n",
    "    tree1 = etree.HTML(test1)\n",
    "    #这里是提取出详细页table[4]下表格表头为(NOC)\n",
    "    r3 = tree1.xpath('/html/body/div[2]/table[4]/thead/tr/th[1]//text()')\n",
    "    #如果提取出来的数据是我们想要的，就获取该页面一级标题\n",
    "    if r3==['NOC']:\n",
    "        ii = tree1.xpath('/html/body/div[2]/h1//text()')\n",
    "        #这里是判断是否有夏季奥运会各个国家获取奖牌数据文件，没有的话就创建该文件，有的话则不创建\n",
    "        if not os.path.exists('./夏季奥运会各个国家获取奖牌数据'  + '/'):\n",
    "            os.makedirs('./夏季奥运会各个国家获取奖牌数据' + '/')\n",
    "        #filename是以一级标题命名，一级标题含有年份以及是夏季奥运会或是冬季奥运会介绍\n",
    "        filename = './夏季奥运会各个国家获取奖牌数据' + '/' + ii[0]\n",
    "        #这里是每个获取出来的数据写入文件中的表头\n",
    "        r_33 = ['NOC', 'abbr', 'Gold', 'Silver', 'Bronze', 'Total']\n",
    "        ll = len(r_33)\n",
    "        i = 0\n",
    "        u1 = 0\n",
    "        #如果表头获取的数据是正确的，那么对应的表格内容也是我们所要提取的\n",
    "        r33 = tree1.xpath('/html/body/div[2]/table[4]//tr//td//text()')\n",
    "        with open(filename+'.csv', 'w', encoding='utf-8')as fp1:\n",
    "            for ll1 in range(ll):\n",
    "                #将r_33列表里的表格头写入文件\n",
    "                fp1.write(str(r_33[ll1]))\n",
    "                u1=u1+1\n",
    "                if u1==6:\n",
    "                    fp1.write('\\n')\n",
    "                else:\n",
    "                    fp1.write(',')\n",
    "            for u in range(len(r33)):\n",
    "                fp1.write(str(r33[u]))\n",
    "                i = i + 1\n",
    "                #表格里的数据一行有六列，那么写入文件也是每六个数据为一行\n",
    "                if i == 6:\n",
    "                    fp1.write('\\n')\n",
    "                    i = 0\n",
    "                else:\n",
    "                    fp1.write(',')\n",
    "    else:\n",
    "        #数据不存放在table[4]表格中，那么判断是否存放在table[5]里，是的就提取出数据\n",
    "        r4 = tree1.xpath('/html/body/div[2]/table[5]/thead/tr/th[1]//text()')\n",
    "        #这里是判断提取出来的表格头是否是不为['Athlete'],不是则为我们需要的数据\n",
    "        if r4!=['Athlete']:\n",
    "            ii1 = tree1.xpath('/html/body/div[2]/h1//text()')\n",
    "            i1 = 0\n",
    "            u1=0\n",
    "            r_44 = ['NOC', 'abbr', 'Gold', 'Silver', 'Bronze', 'Total']\n",
    "            ll=len(r_44)\n",
    "            filename = './夏季奥运会各个国家获取奖牌数据' + '/' + ii1[0]\n",
    "            #上面已经提取出所需要的表头，那么下面就可以提取表格内容，表格内容和表头都是存放在同一个table表下的\n",
    "            r44 = tree1.xpath('/html/body/div[2]/table[5]//tr//td//text()')\n",
    "            with open(filename+'.csv', 'w', encoding='utf-8')as fp1:\n",
    "                for ll1 in range(ll):\n",
    "                    fp1.write(str(r_44[ll1]))\n",
    "                    u1=u1+1\n",
    "                    if u1==6:\n",
    "                        fp1.write('\\n')\n",
    "                        u1=0\n",
    "                    else:\n",
    "                        fp1.write(',')\n",
    "\n",
    "                for u in range(len(r44)):\n",
    "                    fp1.write(str(r44[u]))\n",
    "                    i1 = i1 + 1\n",
    "                    if i1 == 6:\n",
    "                        fp1.write('\\n')\n",
    "                        i1 = 0\n",
    "                    else:\n",
    "                        fp1.write(',')\n",
    "\n",
    "        else:\n",
    "            ii2 = tree1.xpath('/html/body/div[2]/h1//text()')\n",
    "            i2 = 0\n",
    "            u1=0\n",
    "            filename = './夏季奥运会各个国家获取奖牌数据' + '/' + ii2[0]\n",
    "            r_55 = ['NOC', 'abbr', 'Gold', 'Silver', 'Bronze', 'Total']\n",
    "            ll=len(r_55)\n",
    "            with open(filename+'.csv', 'w', encoding='utf-8')as fp1:\n",
    "                for ll1 in range(ll):\n",
    "                    fp1.write(str(r_55[ll1]))\n",
    "                    u1=u1+1\n",
    "                    if u1==6:\n",
    "                        fp1.write('\\n')\n",
    "                        u1=0\n",
    "                    else:\n",
    "                        fp1.write(',')\n",
    "                r55 = tree1.xpath('/html/body/div[2]/table[3]//tr//td//text()')\n",
    "                for u in range(len(r55)):\n",
    "                    fp1.write(str(r55[u]))\n",
    "                    i2 = i2 + 1\n",
    "                    if i2 == 6:\n",
    "                        fp1.write('\\n')\n",
    "                        i2 = 0\n",
    "                    else:\n",
    "                        fp1.write(',')"
   ],
   "id": "dc75af8f778f1fac",
   "outputs": [],
   "execution_count": 7
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
